#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Datetime: 2021/11/30 16:18
# @Author  : CHEN Wang
# @Site    : 
# @File    : coinglass.py
# @Software: PyCharm

"""
脚本说明:
Some endpoints will require an API Key. Please refer to this page.
The basbe endpoinpus: https://open-api.coinglass.com
All endpoints return either a JSON object or array.

symbol:
BTC,ETH,EOS,BCH,LTC,XRP,BSV,ETC,TRX,LINK are now supported

Exchange Name:
Bitmex,Binance,Bybit,Okex,Huobi,FTX,Deribit,Kraken,Bitfinex,Phemex are now supported

All api require permission validation.you need to add "coinglassSecret" in http request header.
"""

import time
import numpy as np
import pandas as pd
import requests
import json
from retrying import retry
import os
from quant_researcher.quant.project_tool.time_tool import get_today, calc_date_diff
from quant_researcher.quant.project_tool.localize import DATA_DIR

api_key = '04c8f778c4124f909fcea0df1d0cb992'


@retry(stop_max_attempt_number=3)
def get_recent_funding_rate(asset='BTC', market_type='C', **kwargs):
    """

    :param str asset: 资产类别，默认为'BTC'， BTC,ETH,EOS,BCH,LTC,XRP,BSV,ETC,TRX,LINK are now supported
    (Token Margined=C, USDT or USD Margined=U)
    :param kwargs: 关键字参数
    :return:
    """

    header = {
        'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36',
        'coinglassSecret': api_key
    }

    # url = f'https://open-api.coinglass.com/api/pro/v1/futures/funding_rates_chart?symbol={asset}&type={market_type}'  # 该版本api已经不能用了
    if market_type == 'C':
        url = f"https://open-api.coinglass.com/public/v2/funding_coin_history?symbol={asset}&time_type=h8"
    else:
        url = f"https://open-api.coinglass.com/public/v2/funding_usd_history?symbol={asset}&time_type=h8"

    response = requests.request("GET", url, headers=header, timeout=15)
    res = json.loads(response.text)
    if 'data' not in res.keys():
        print(f'指标{asset}-{market_type}最新资金费率数据获取失败， 错误提示{res["msg"]}')
        return None
    else:
        res = res['data']
    if res['dateList']:
        res_list = []
        for item_key in ['dateList', 'frDataMap', 'priceList']:
            if item_key == 'frDataMap':
                for item in res[item_key]:
                    list_temp = res[item_key][item]
                    res_list.append(list_temp)
            else:
                list_temp = res[item_key]
                res_list.append(list_temp)
        df = pd.DataFrame(res_list).T
        df.columns = ['timestamp'] + list(res['frDataMap'].keys()) + ['price']

        print(f'指标{asset}-{market_type}最新资金费率数据获取成功')
        return df
    else:
        print(f'指标{asset}-{market_type}最新资金费率数据为空，无数据')
        return None


@retry(stop_max_attempt_number=3)
def get_gbtc_premium(**kwargs):
    """

    :param kwargs: 关键字参数
    :return:
    """

    header = {
        'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36',
        'coinglassSecret': api_key
    }

    url = f'https://fapi.coinglass.com/api/grayscale/market/history?symbol=BTC'

    response = requests.request("GET", url, headers=header, timeout=15)
    res = json.loads(response.text)['data']
    res_list = []
    for item_key in ['dateList', 'premiumRateList']:
        list_temp = res[item_key]
        res_list.append(list_temp)
    df = pd.DataFrame(res_list).T
    df.columns = ['timestamp', 'gbtc_premium']

    df['date'] = pd.to_datetime(df['timestamp'], unit='ms').dt.tz_localize('UTC').dt.tz_convert('+0000')
    df['date'] = df['date'].dt.strftime('%Y-%m-%d')
    df.set_index('date', inplace=True)

    print(f'gbtc_premium指标最新数据获取成功')
    return df


@retry(stop_max_attempt_number=3)
def get_exchange_open_interest(asset='BTC', interval=0):
    """
    interval	int	(0=ALL, 2=1H ,1=4H 4=12H)
    symbol	string	Symbol
    :return:
    """

    header = {
        'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36',
        'coinglassSecret': api_key
    }

    url = f'https://open-api.coinglass.com/api/pro/v1/futures/openInterest?symbol={asset}&interval={interval}'

    response = requests.request("GET", url, headers=header, timeout=15)
    res = json.loads(response.text)['data']

    df = pd.DataFrame(res)

    print(f'指标{asset}最新各交易所期货开仓量获取成功')
    return df


@retry(stop_max_attempt_number=3)
def get_exchange_open_interest_chart(asset='BTC', interval=0):
    """

    interval	int	(0=ALL, 2=1H ,1=4H 4=12H)
    symbol	string	Symbol
    :return:
    """

    header = {
        'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36',
        'coinglassSecret': api_key
    }

    url = f'https://open-api.coinglass.com/api/pro/v1/futures/openInterest/chart?symbol={asset}&interval={interval}'

    response = requests.request("GET", url, headers=header, timeout=15)
    res = json.loads(response.text)['data']

    res_list = []
    for item_key in ['dateList', 'dataMap', 'priceList']:
        if item_key == 'dataMap':
            for item in res[item_key]:
                list_temp = res[item_key][item]
                res_list.append(list_temp)
        else:
            list_temp = res[item_key]
            res_list.append(list_temp)
    df = pd.DataFrame(res_list).T
    df.columns = ['timestamp'] + list(res['dataMap'].keys()) + ['price']

    timezone = '+0000'
    df['timestamp'] = df.apply(lambda x: int(x['timestamp'] / 1000), axis=1)
    df['date'] = pd.to_datetime(df['timestamp'], unit='s').dt.tz_localize('UTC').dt.tz_convert(timezone)
    df['date'] = df['date'].dt.strftime('%Y-%m-%d')
    df['datetime'] = pd.to_datetime(df['timestamp'], unit='s').dt.tz_localize('UTC').dt.tz_convert(timezone)
    df['datetime'] = df['datetime'].dt.strftime('%Y-%m-%d %H:%M:%S')
    df['time'] = pd.to_datetime(df['timestamp'], unit='s').dt.tz_localize('UTC').dt.tz_convert(timezone)
    df['time'] = df['time'].dt.strftime('%H:%M:%S')
    df.set_index('date', inplace=True)

    print(f'指标{asset}各交易所历史期货开仓量获取成功')
    return df


@retry(stop_max_attempt_number=3)
def get_exchange_liquidation(asset='BTC', exchange='Binance'):
    """

    :return:
    """

    header = {
        'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36',
        'coinglassSecret': api_key
    }

    url = f'https://open-api.coinglass.com/api/pro/v1/futures/liquidation_chart?symbol={asset}&exName={exchange}'

    response = requests.request("GET", url, headers=header, timeout=15)
    res = json.loads(response.text)['data']

    res_list = []
    for item_key in ['dateList', 'priceList', 'numList', 'volList', 'sellList', 'buyList']:
        list_temp = res[item_key]
        res_list.append(list_temp)
    df = pd.DataFrame(res_list).T
    df.columns = ['timestamp', 'price', 'num', 'vol', 'short', 'long']

    timezone = '+0000'
    df['timestamp'] = df.apply(lambda x: int(x['timestamp'] / 1000), axis=1)
    df['date'] = pd.to_datetime(df['timestamp'], unit='s').dt.tz_localize('UTC').dt.tz_convert(timezone)
    df['date'] = df['date'].dt.strftime('%Y-%m-%d')
    df['datetime'] = pd.to_datetime(df['timestamp'], unit='s').dt.tz_localize('UTC').dt.tz_convert(timezone)
    df['datetime'] = df['datetime'].dt.strftime('%Y-%m-%d %H:%M:%S')
    df['time'] = pd.to_datetime(df['timestamp'], unit='s').dt.tz_localize('UTC').dt.tz_convert(timezone)
    df['time'] = df['time'].dt.strftime('%H:%M:%S')
    df.set_index('datetime', inplace=True)

    print(f'指标{asset}多头空头被清算量获取成功')
    return df


@retry(stop_max_attempt_number=3)
def get_exchange_liquidation_chart(asset='BTC', interval=18):
    """

    timeType	int	(1m=9, 5m=3, 15m=10, 30=11, 4h=1, 12h=4, 90d =18)
    symbol	string	Symbol

    :return:
    """

    header = {
        'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36',
        'coinglassSecret': api_key
    }

    url = f'https://open-api.coinglass.com/api/pro/v1/futures/liquidation/detail/chart?symbol={asset}&timeType={interval}'

    response = requests.request("GET", url, headers=header, timeout=15)
    res = json.loads(response.text)['data']
    df = pd.DataFrame(res)

    timezone = '+0000'
    df['timestamp'] = df.apply(lambda x: int(x['createTime'] / 1000), axis=1)
    df['date'] = pd.to_datetime(df['timestamp'], unit='s').dt.tz_localize('UTC').dt.tz_convert(timezone)
    df['date'] = df['date'].dt.strftime('%Y-%m-%d')
    df['datetime'] = pd.to_datetime(df['timestamp'], unit='s').dt.tz_localize('UTC').dt.tz_convert(timezone)
    df['datetime'] = df['datetime'].dt.strftime('%Y-%m-%d %H:%M:%S')
    df['time'] = pd.to_datetime(df['timestamp'], unit='s').dt.tz_localize('UTC').dt.tz_convert(timezone)
    df['time'] = df['time'].dt.strftime('%H:%M:%S')
    df.set_index('datetime', inplace=True)

    return df


@retry(stop_max_attempt_number=3)
def get_longshort_chart(asset='BTC', interval=5):
    """

    interval	int	(1h=2, 4h=1, 12h=4, 24h=5)
    symbol	string	Symbol

    :return:
    """

    header = {
        'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36',
        'coinglassSecret': api_key
    }

    url = f'https://open-api.coinglass.com/api/pro/v1/futures/longShort_chart?symbol={asset}&interval={interval}'

    response = requests.request("GET", url, headers=header, timeout=15)
    res = json.loads(response.text)['data']
    res_list = []
    for item_key in ['dateList', 'priceList', 'longShortRateList']:
        list_temp = res[item_key]
        res_list.append(list_temp)
    df = pd.DataFrame(res_list).T
    df.columns = ['timestamp', 'price', 'longshort_ratio']

    timezone = '+0000'
    df['timestamp'] = df.apply(lambda x: int(x['timestamp'] / 1000), axis=1)
    df['date'] = pd.to_datetime(df['timestamp'], unit='s').dt.tz_localize('UTC').dt.tz_convert(timezone)
    df['date'] = df['date'].dt.strftime('%Y-%m-%d')
    df['datetime'] = pd.to_datetime(df['timestamp'], unit='s').dt.tz_localize('UTC').dt.tz_convert(timezone)
    df['datetime'] = df['datetime'].dt.strftime('%Y-%m-%d %H:%M:%S')
    df['time'] = pd.to_datetime(df['timestamp'], unit='s').dt.tz_localize('UTC').dt.tz_convert(timezone)
    df['time'] = df['time'].dt.strftime('%H:%M:%S')
    df.set_index('datetime', inplace=True)

    return df


def get_all_funding_rate():
    file_path = os.path.join(DATA_DIR, r'funding_rate\all_market_funding_rate_coinglass')
    os.makedirs(file_path, exist_ok=True)
    end_date = get_today(marker='with_n_dash')
    symbol_list = ['BTC', 'ETH', 'XRP', 'BNB', 'LTC', 'DOGE', 'SOL', 'APT', 'MATIC', 'ADA', 'DOT', 'FIL', 'OP',
                   'USDC', 'LINK', 'ETC', 'BCH', 'AVAX', 'EOS', 'CFX', 'APE', 'FTM', 'ATOM', 'MASK', 'STX', 'LDO',
                   'DYDX', 'TRX', 'GALA', 'GMT', 'NEAR', 'AGIX', 'SAND', 'CRV', 'SNX', 'UNI', 'AXS', 'MAGIC', 'MANA',
                   '1000SHIB', 'BLUR', 'GRT', 'MKR', 'IMX', 'SUSHI', 'CHZ', 'AAVE', 'XMR', 'GMX', 'LQTY', 'DASH',
                   'ICP', 'XLM', 'ALGO', 'ENS', '1000LUNC', 'KAVA', 'SHIB', 'FET', 'EGLD', 'KLAY', 'WAVES', 'ZEC',
                   'NEO', 'ARPA', 'YFI', 'PEOPLE', 'XTZ', 'AR', 'HOOK', 'THETA', 'BNX', 'TOMO', 'INJ', 'ROSE', 'VET',
                   'ANKR', 'LRC', 'BIT', 'OCEAN', 'LUNA2', 'KSM', 'ZIL', 'BTCDOM', 'RUNE', 'COMP', 'RNDR', 'REN',
                   'TRU', 'MINA', '1INCH', 'STG', 'FLOW', 'SSV', 'ENJ', 'FXS', 'HNT', 'TRB', 'GAL', 'JASMY', 'QTUM',
                   'LUNC', 'ZEN', 'HEAR', 'WOO', 'KNC', 'CELR', 'SXP', 'RSR', 'CELO', 'ACH', 'LIT', 'HIGH', 'SHIB1000',
                   'UNFI', 'IOST', 'LPT', 'DUSK', 'BSV', 'ONE', 'BAND', 'AUDIO', 'RVN', 'ZRX', 'IOTA', 'ATA', 'SFP',
                   'STORJ', 'C98', 'PHB', 'ALICE', 'OMG', 'COTI', 'CORE', 'XEM', '1000BONK', 'BAL', 'ALPHA', 'ASTR',
                   'LINA', 'COCOS', 'BAT', 'BEL', 'ONT', 'CHR', 'REEF', 'LUNA', 'MTL', 'BLZ', 'HOT', 'BAKE', 'DENT',
                   'SKL', 'GTC', 'ETHW', 'IOTX', 'FLM', 'CTK', 'TWT', 'ICX', 'CRO', 'OGN', 'CVX', 'API3', 'DGB', 'ANT',
                   'RLC', 'DODO', 'STMX', 'CTSI', 'QNT', 'NKN', 'DAR', 'YFII', 'BUSD', 'LOOKS', 'CKB', 'PERP', 'TON', 'GFT',
                   'SPELL', 'YGG', 'USTC', 'GPT', 'PAXG', 'FITFI', '1000XEC', 'DEFI', 'SLP', 'HFT', 'TLM', 'CVC', 'ILV',
                   'FLOKI', 'HT', 'GLMR', 'XCH', 'USDT', 'JST', 'CEL', 'BICO', 'AGLD', 'BNT', 'KDA', 'BLUEBIRD', 'CREAM',
                   'SWEAT', 'UMA', 'BSW', 'BMEX', 'XNO', 'FLR']
    timezone = '+0000'

    all_data_df_list = []
    all_cm_data_df_list = []
    all_um_data_df_list = []
    for symbol in symbol_list:
        for market_type in ['U', 'C']:
            file_name = os.path.join(file_path, f'coinglass_{symbol}_{market_type}_funding_rate')

            if os.path.exists(f'{file_name}.xlsx'):
                history_funding_rate = pd.read_excel(f'{file_name}.xlsx', index_col='datetime')
                start_date = history_funding_rate.index[-1]
                if calc_date_diff(start_date[:10], end_date) < 3:
                    print(f'{symbol}-{market_type}资金费率数据已经最新')
                    history_funding_rate = history_funding_rate[[i for i in history_funding_rate.columns if i not in ['timestamp', 'price', 'date', 'time']]]
                    history_funding_rate.columns = [i + f'_{symbol}_{market_type}' for i in history_funding_rate.columns]
                    all_data_df_list.append(history_funding_rate)
                    if market_type == 'U':
                        all_um_data_df_list.append(history_funding_rate)
                    elif market_type == 'C':
                        all_cm_data_df_list.append(history_funding_rate)
                    else:
                        raise NotImplementedError
                    continue
            else:
                history_funding_rate = pd.DataFrame()

            recent_df = get_recent_funding_rate(asset=symbol, market_type=market_type)

            if recent_df is None:
                print(f'{symbol}-{market_type}资金费率最新数据为空，没有数据')
                continue
            else:
                recent_df['timestamp'] = recent_df.apply(lambda x: int(x['timestamp'] / 1000), axis=1)
                recent_df['date'] = pd.to_datetime(recent_df['timestamp'], unit='s').dt.tz_localize('UTC').dt.tz_convert(timezone)
                recent_df['date'] = recent_df['date'].dt.strftime('%Y-%m-%d')
                recent_df['datetime'] = pd.to_datetime(recent_df['timestamp'], unit='s').dt.tz_localize('UTC').dt.tz_convert(timezone)
                recent_df['datetime'] = recent_df['datetime'].dt.strftime('%Y-%m-%d %H:%M:%S')
                recent_df['time'] = pd.to_datetime(recent_df['timestamp'], unit='s').dt.tz_localize('UTC').dt.tz_convert(timezone)
                recent_df['time'] = recent_df['time'].dt.strftime('%H:%M:%S')
                recent_df.set_index('datetime', inplace=True)
                recent_df = recent_df.iloc[:-1, ]  # 最后一条数据不完整，需要剔除

                all_df = pd.concat([history_funding_rate, recent_df], axis=0)
                all_df = all_df[~all_df.index.duplicated(keep='first')]  # index去重

                all_df.to_excel(f'{file_name}.xlsx')

                all_df = all_df[[i for i in all_df.columns if i not in ['timestamp', 'price', 'date', 'time']]]
                all_df.columns = [i + f'_{symbol}_{market_type}' for i in all_df.columns]
                all_data_df_list.append(all_df)
                if market_type == 'U':
                    all_um_data_df_list.append(all_df)
                elif market_type == 'C':
                    all_cm_data_df_list.append(all_df)
                else:
                    raise NotImplementedError
                time.sleep(10)

    all_funding_rate_df = pd.concat(all_data_df_list, axis=1)
    all_cm_funding_rate_df = pd.concat(all_cm_data_df_list, axis=1)
    all_um_funding_rate_df = pd.concat(all_um_data_df_list, axis=1)
    all_funding_rate_df.sort_index(inplace=True)
    all_cm_funding_rate_df.sort_index(inplace=True)
    all_um_funding_rate_df.sort_index(inplace=True)

    temp_index = [i for i in all_funding_rate_df.index if i[11:13] in ['00', '08', '16']]  # 只保留三个时间点的
    all_funding_rate_df = all_funding_rate_df.loc[temp_index, :]
    temp_index = [i for i in all_cm_funding_rate_df.index if i[11:13] in ['00', '08', '16']]  # 只保留三个时间点的
    all_cm_funding_rate_df = all_cm_funding_rate_df.loc[temp_index, :]
    temp_index = [i for i in all_um_funding_rate_df.index if i[11:13] in ['00', '08', '16']]  # 只保留三个时间点的
    all_um_funding_rate_df = all_um_funding_rate_df.loc[temp_index, :]

    temp_file_path = os.path.join(DATA_DIR, f'sth_momentum')
    temp_file_name = os.path.join(temp_file_path, f'glassnode_btcusdt_hourly_ohlcv')
    hourly_ohlcv = pd.read_excel(f'{temp_file_name}.xlsx', index_col='end_date')

    for index, df in enumerate([all_funding_rate_df, all_cm_funding_rate_df, all_um_funding_rate_df]):
        df = df.astype(float)
        analysis_df = pd.DataFrame()
        analysis_df['total_num'] = (~df.isnull()).sum(axis=1)
        analysis_df['neutral_num'] = (df == 0.01).sum(axis=1)
        analysis_df['positive_num'] = (df > 0.01).sum(axis=1)
        analysis_df['negative_num'] = (df < 0.01).sum(axis=1)
        analysis_df['neutral_ratio'] = analysis_df['neutral_num'] / analysis_df['total_num']
        analysis_df['positive_ratio'] = analysis_df['positive_num'] / analysis_df['total_num']
        analysis_df['negative_ratio'] = analysis_df['negative_num'] / analysis_df['total_num']
        analysis_df['neutral_ratio_ma7'] = analysis_df['neutral_ratio'].rolling(7).mean()
        analysis_df['positive_ratio_ma7'] = analysis_df['positive_ratio'].rolling(7).mean()
        analysis_df['negative_ratio_ma7'] = analysis_df['negative_ratio'].rolling(7).mean()

        all_analysis_df = analysis_df.merge(hourly_ohlcv, how='left', left_index=True, right_index=True)

        if index == 0:
            file_name = os.path.join(file_path, f'all_market_funding_rate_analysis')
        elif index == 1:
            file_name = os.path.join(file_path, f'all_cm_funding_rate_analysis')
        elif index == 2:
            file_name = os.path.join(file_path, f'all_um_funding_rate_analysis')
        else:
            raise NotImplementedError
        all_analysis_df.to_excel(f'{file_name}.xlsx')


if __name__ == '__main__':
    # gbtc_premium = get_gbtc_premium()
    # file_name = os.path.join(DATA_DIR, f'coinglass_gbtc_premium')
    # gbtc_premium.to_excel(f'{file_name}.xlsx')

    # df = get_exchange_open_interest_chart(asset='BTC', interval=0)

    # df = get_exchange_liquidation()
    # get_exchange_liquidation_chart()
    get_all_funding_rate()
    # get_longshort_chart()
